﻿//
// Copyright 2008-2011 by Burt Harris
// Licensed under the Microsoft Public License (Ms-PL)
// See: http://www.codeplex.com/yaml/license
//

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.Globalization;
using System.Text;
using System.Text.RegularExpressions;
using Microsoft.Scripting;
using System.Diagnostics.Contracts;
using System.IO;

namespace YaTools.Yaml
{
    /// <summary>A Yaml 1.2 parser</summary>
    /// <remarks>
    /// This parser is an abstract class with virtual On*() functions which must be overridden
    /// to create a concrete, functional parser.   
    /// </remarks>
    [DebuggerDisplay("{DebugLookahead}")]
    [ContractVerification(false)]
    public class YamlParser : IDisposable
    {
        /// <summary>
        /// Controls tracing of per-token level events
        /// </summary>
        public static readonly TraceSwitch TraceTokens =
            new TraceSwitch("YamlParser.TraceTokens",
                            "YamlParser tokens", "Error");

        public static readonly TraceSwitch TraceParse =
            new TraceSwitch("YamlParser.TraceParse",
                            "YamlParser tracing", "Error");

        /// <summary>
        /// Regex used to parse YAML directives
        /// </summary>
        private static readonly Regex YamlDirectiveRegex =
            new Regex(@"^%YAML\s+(\d+\.\d+)\s*$");

        /// <summary>
        /// Regex use to parse TAG directives
        /// </summary>
        private static readonly Regex TagDirectiveRegex =
            new Regex(@"^%TAG\s+(!\w*!?) +(\S+)\s*$");

        private readonly Load _load;
        private readonly StringBuilder _lineBreaks = new StringBuilder(); // Accumulates breaks prior to chomping
        private readonly StringBuilder _scalarBuilder = new StringBuilder(); // Accumulates scalar content

        /// <summary>
        /// Scanner
        /// </summary>
        /// <remarks>
        /// The parser depends on YamlScanner to tokenize the input, and take care of lookahead 
        /// requirements (e.g. literal keys.), allowing this class to take a simple 
        /// recursive-decent approach.
        /// </remarks>
        private readonly YamlScanner _scanner;

        /// <summary>
        /// Cached mappings of tags shorthand to Tag.   Cleared at the end of each document.
        /// </summary>
        private readonly Dictionary<string, Tag> _shortcutCache = new Dictionary<string, Tag>();

        /// <summary>
        /// Current mapping of tag-handles to tag-prefixes
        /// </summary>
        private readonly Dictionary<string, string> _tagDocumentMap = new Dictionary<string, string>();

        /// <summary>
        /// During header processing, new mappings from tag-handles to tag-prefixes
        /// created by %TAG directives.
        /// </summary>
        private readonly Dictionary<string, string> _tagHeaderMap = new Dictionary<string, string>();

        /// <summary>
        /// True only for first document in a string.
        /// </summary>
        private bool _firstDocumentInStream = true;

        private int _longestEmptyLine;

        private string _normalizedNewLine = "\n";

        /// <summary>
        /// Most recently scanned token
        /// </summary>
        private Token _token = Token.BeginningOfStream;

        /// <summary>
        /// True if this document already has a YAML directive.
        /// </summary>
        private bool _yamlVersionSetThisDocument;

        /// <summary>
        /// Creates a YamlParser with a specified sink instance
        /// </summary>
        public YamlParser(Compose sink)
            : this(new YamlScanner(), new Load(sink))
        {
        }

        [CLSCompliant(false)]
        protected YamlParser(YamlScanner scanner, Load load)
        {
            Contract.Requires(scanner != null);
            Contract.Requires(load != null);
            _load = load;
            _scanner = scanner;
            // Pre-populate default tag handle mappings
            _tagDocumentMap.Add("!", "!");
            _tagDocumentMap.Add("!!", "tag:yaml.org,2002:"); // tag:yaml.org,2002
            NormalizedNewLine = "\n";
            _scanner = scanner;
        }

        internal Load Load
        {
            get { return _load; }
        }

        [ContractInvariantMethod]
        private void ObjectInvariant()
        {
            Contract.Invariant(_scanner != null);
            Contract.Invariant(_load != null);
            Contract.Invariant(_lineBreaks != null);
            Contract.Invariant(_tagHeaderMap != null);
            Contract.Invariant(_shortcutCache != null);
            Contract.Invariant(_scalarBuilder != null);
            Contract.Invariant(_tagDocumentMap != null);
        }

        /// <summary>
        /// YAML version number from last YAML directive, 
        /// empty if no YAML directive so far in the stream.
        /// </summary>
        public string YamlVersion { get; private set; }

        public string NormalizedNewLine
        {
            get { return _normalizedNewLine; }
            set
            {
                switch (value)
                {
                    case "\n":
                    case "\r\n":
                    case "\r":
                        _normalizedNewLine = value;
                        break;
                    default:
                        throw new ArgumentException("Unsupported newline type", value);
                }
            }
        }

        public ErrorSink ErrorSink
        {
            get { return _scanner.ErrorSink; }
            set
            {
                Contract.Requires(value != null);
                _scanner.ErrorSink = value;
            }
        }

        [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")]
        // ReSharper disable UnusedMember.Local
            private string DebugLookahead
        {
            get { return _scanner.DebugLookahead; }
        }

        // ReSharper restore UnusedMember.Local

        private Token Token
        {
            [DebuggerStepThrough]
            get
            {
                if (_token == Token.BeginningOfStream)
                    NextToken();
                return _token;
            }
        }

        public string TokenText
        {
            get { return _scanner.TokenText; }
        }

        public SourceSpan TokenSpan
        {
            get { return _scanner.TokenSpan; }
        }

        public SourceLocation TokenStart
        {
            get { return _scanner.TokenSpan.Start; }
        }

        // ReSharper disable UnusedMember.Local
        private string DebugToken
        {
            get { return _scanner.DebugToken; }
        }

        // ReSharper restore UnusedMember.Local

        [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")]
        // ReSharper disable UnusedMember.Local
            private string DebugScalar
        {
            get { return Utility.VisibleSpacing(_scalarBuilder.ToString()); }
        }

        // ReSharper restore UnusedMember.Local

        protected void SetSink(Compose sink)
        {
            _load.Initialize(sink);
        }

        public void SetSource(TextReader reader)
        {
            Contract.Requires(reader != null);

            _scanner.SetSource(reader);
        }

        public void SetSource(string yamlText)
        {
            Contract.Requires(yamlText != null);
            _scanner.SetSource(yamlText);
        }

        protected bool AddError(ErrorId errorId)
        {
            _scanner.AddError(errorId, _scanner.TokenText);
            return false;
        }

        protected bool AddError(ErrorId errorCode, SourceSpan span)
        {
            _scanner.AddError(errorCode, span, Severity.Error);
            return false;
        }

        protected bool AddError(Severity severity, ErrorId errorCode, SourceSpan span)
        {
            _scanner.AddError(severity, errorCode, span);
            return false;
        }

        protected void IssueWarning(ErrorId errorCode)
        {
            _scanner.AddError(Severity.Warning, errorCode, _scanner.TokenText);
        }

        // Debugging aid, to make watch window simpler:

        private Token NextRawToken()
        {
            while (true)
            {
                _token = _scanner.NextToken();
                if (TraceTokens.TraceVerbose)
                    Trace.WriteLine(Token, "NextToken");
                if (_token != Token.IndentSpaces)
                    return _token;
            }
        }

        private void NextToken()
        {
            while (true)
            {
                switch (_token = _scanner.NextToken())
                {
                    default:
                        if (TraceTokens.TraceInfo)
                        {
                            Trace.WriteLine(
                                String.Format(CultureInfo.InvariantCulture,
                                              "Parser Token: {0} <{2}> {1}",
                                              _token, _scanner.TokenSpan,
                                              Utility.DebugDisplay(TokenText)));
                        }
                        return;

                    case Token.Newline:
                    case Token.IndentSpaces:
                    case Token.Comment:
                        continue;
                }
            }
        }

        /// <summary>
        /// Called when parser encounters an Anchor
        /// </summary>
        /// <param name="anchor">Anchor name, including the '&'</param>
        protected virtual void OnAnchor(string anchor)
        {
            if (TraceParse.TraceVerbose)
                Debug.WriteLine(anchor, _scanner.TokenSpan + "\tOnAnchor");
            _load.Anchor = anchor;
        }

        /// <summary>
        /// Called when parsing an directives-end indicator
        /// </summary>
        /// <remarks>
        /// Triggered by "---" at beginning of a line.   
        /// 
        /// In Yaml 1.1, this was called a document-start indicator, this is a better name.
        /// </remarks>
        private void OnDirectivesEnd()
        {
            if (TraceParse.TraceVerbose)
                Debug.WriteLine(TokenText, _scanner.TokenSpan.Start + "\tOnDirectivesEnd");

            // Move pending tag-handle to tag-prefix entries into main mapping, 
            // preserving any non-overridden values across documents.
            foreach (string key in _tagHeaderMap.Keys)
                _tagDocumentMap[key] = _tagHeaderMap[key];

            _tagHeaderMap.Clear();
        }

        /// <summary>
        /// Called when parsing a directive.
        /// </summary>
        /// <remarks>
        /// YamlParser handles the two directives allowed in the YAML 1.2 specification,
        /// you would only override this to violate the explicitly stated non-extensibility of 
        /// directives.
        /// </remarks>
        /// <param name="directive"></param>
        private void OnDirective(string directive)
        {
            Contract.Assume(directive != null);
            if (TraceParse.TraceVerbose)
                Debug.WriteLine(directive, _scanner.TokenSpan.Start + "\tOnDirective");
            Match match = YamlDirectiveRegex.Match(directive);
            if (match.Success)
            {
                OnYamlDirective(match.Groups[1].Value);
                return;
            }
            match = TagDirectiveRegex.Match(directive);
            if (match.Success)
            {
                OnTagDirective(match.Groups[1].Value, match.Groups[2].Value);
                return;
            }
            IssueWarning("YA002");
        }

        // YAML Directive handling (private for now)
        private void OnYamlDirective(string version)
        {
            Contract.Requires(version != null);

            if (_yamlVersionSetThisDocument)
                AddError("YA304");

            switch (version)
            {
                case "1.1":
                case "1.2":
                    break;
                default:
                    IssueWarning("YA001");
                    break;
            }
            YamlVersion = version;
            _yamlVersionSetThisDocument = true;
        }

        private void OnTagDirective(string handle, string prefix)
        {
            Contract.Requires(!string.IsNullOrEmpty(handle));
            Contract.Requires(!string.IsNullOrEmpty(prefix));

            // ValidateBuffer the handle
            if (!IsValidTagHandle(handle))
            {
                AddError("YA306");
                return;
            }

            // ValidateBuffer the prefix
            if (!Tag.IsValidTagPrefix(prefix))
            {
                AddError("YA307");
                return;
            }

            // Save TAG in working dictionary

            if (_tagHeaderMap.ContainsKey(handle))
                AddError("YA305");

            _tagHeaderMap.Add(handle, prefix);
        }

        private static bool IsValidTagHandle(string handle) // [89]
        {
            if (String.IsNullOrEmpty(handle)) return false;
            if (handle[0] != '!') return false;
            if (handle[handle.Length - 1] != '!') return false;
            for (int i = 1; i < handle.Length - 2; i++)
                if (!(Char.IsLetterOrDigit(handle, i) || handle[i] == '-'))
                    return false;
            return true;
        }


        /// <summary>
        /// Translate a tag to a tag-uri
        /// </summary>
        /// <returns>URI for tag, or local-uri (starting with a '!')</returns>
        private Tag ShortcutToTag(string shortcut, Kind kind)
        {
            const StringComparison ordinal = StringComparison.Ordinal;

            Contract.Assume(!string.IsNullOrEmpty(shortcut));
            Contract.Assume(shortcut.Length >= 1 && shortcut[0] == '!', "shortcut must start with '!'");
            if (shortcut.Length == 1) return Tag.NonSpecificNotPlainScalar;

            Tag tag;

            if (_shortcutCache.TryGetValue(shortcut, out tag)) return tag;

            if (shortcut.StartsWith("!<", ordinal) && shortcut.EndsWith(">", ordinal))
            {
                Contract.Assume(shortcut.Length >= 3);
                return AddShortcutToCache(shortcut, 
                    new Tag(shortcut.Substring(2, shortcut.Length - 3), kind));
            }
            int handleLength = shortcut.IndexOf('!', 1) + 1;
            if (handleLength == 0)
                return Tag.NonSpecificNotPlainScalar;

            string handle = shortcut.Substring(0, handleLength);
            string suffix = shortcut.Substring(handle.Length);
            string prefix;

            if (_tagDocumentMap.TryGetValue(handle, out prefix))
                return AddShortcutToCache(shortcut, new Tag(prefix + suffix, kind));

            AddError("YA308");
            return Tag.NonSpecificNotPlainScalar;
        }

        private Tag AddShortcutToCache(string shortcut, Tag tag)
        {
            Contract.Requires(shortcut != null);
            _shortcutCache.Add(shortcut, tag);
            return tag;
        }

        //private void ValidateTag(string tag)
        //{
        //    Contract.Requires(!string.IsNullOrEmpty(tag));

        //    if (!Tag.IsValidTag(tag))
        //        AddError("YA307");
        //    return;
        //}

        // FLOW STYLE PRODUCTIONS  (Chapter 7)

        private bool ParseAliasNode() // [104]
        {
            if (Token != Token.Alias) return false;
            _load.AddAlias(TokenText);
            NextToken();
            return true;
        }

        private bool ParseEmptyScalar() // [105] 
        {
            _load.AddScalar(String.Empty);
            return true;
        }

        private bool ParseEmptyNode() // [106]
        {
            ConsumeOutdent();
            return ParseEmptyScalar();
        }

        private bool ParseDoubleQuoted() // [109] (107-116)
        {
            if (Token != Token.DoubleQuote) return false;
            return ParseFlowScalar(Token.DoubleQuote);
        }

        private bool ParseSingleQuoted() // [120] (117-125)
        {
            if (Token != Token.SingleQuote) return false;
            return ParseFlowScalar(Token.SingleQuote);
        }

        private bool ParsePlain() // [131] (...)
        {
            if (Token != Token.PlainScalar) return false;
            return ParseFlowScalar(Token.PlainEnd);
        }

        private bool ParseFlowSequence() // [137]
        {
            if (Token != Token.OpenBracket) return false;
            SourceSpan startSpan = _scanner.TokenSpan;
            _load.BeginSeq(null);
            NextToken();
            ParseFlowSeqEntries();
            if (Token != Token.CloseBracket)
            {
                AddError("YA113", new SourceSpan(startSpan.Start, _scanner.TokenEnd));
                AddError("YA115", _scanner.TokenSpan);
            }
            _load.EndSequenceNode();
            NextToken();
            return true;
        }

        private void ParseFlowSeqEntries() // [138]
        {
            while (ParseFlowSeqEntry())
            {
                if (!MatchToken(Token.Comma))
                    break;
            }
            return;
        }

        private bool ParseFlowSeqEntry() // [139] & [150}
        {
            switch (Token)
            {
                case Token.ImplicitKey:
                case Token.ValueIndicator:
                    _load.BeginMap(null);
                    try
                    {
                        ParseFlowPairEntry();
                    }
                    finally
                    {
                        _load.EndMappingNode();
                    }
                    return true;

                case Token.FlowKeyIndicator:
                    _load.BeginMap(null);
                    NextToken();
                    ParseFlowMapExplicitEntry();
                    _load.EndMappingNode();
                    return true;

                default:
                    return ParseFlowNode();
            }
        }

        /// <summary>Parse Properties</summary>
        /// <returns>True if properties found, false otherwise</returns>
        private bool ParseProperties() // [96]-[103]
        {
            string shortcut = null;

            _load.Tag = Tag.NonSpecific; 

            switch (Token)
            {
                case Token.Anchor:
                    OnAnchor(TokenText);
                    NextToken();
                    if (Token == Token.Tag)
                    {
                        shortcut = TokenText;
                        NextToken();
                    }
                    break;

                case Token.Tag:
                    shortcut = TokenText;
                    NextToken();
                    if (Token == Token.Anchor)
                    {
                        OnAnchor(TokenText);
                        NextToken();
                    }
                    break;

                default:
                    return false;
            }

            while (Token == Token.Anchor || Token == Token.Tag)
            {
                AddError("YA122");
                NextToken();
            }

            if (Token == Token.Alias)
            {
                AddError("YA123");
                NextToken();
            }

            // Perhaps we're in a position to combine the tag with the Kind now...

            if (shortcut != null)
            {
                if ((Token == Token.BlockMappingBegin || Token == Token.OpenBrace))
                    _load.Tag = ShortcutToTag(shortcut, Kind.Mapping);
                else if (Token == Token.BlockSeqBegin || Token == Token.OpenBracket)
                    _load.Tag = ShortcutToTag(shortcut, Kind.Sequence);
                else
                    _load.Tag = ShortcutToTag(shortcut, Kind.Scalar);
            }
            
            return true;
        }

        /// <summary>
        /// Parses an anchor property, or returns false.
        /// </summary>
        private void ParseAnchorProperty() // [101]-[103]
        {
            if (Token != Token.Anchor) return;
            OnAnchor(TokenText);
            NextToken();
            return;
        }

        /// <summary>
        /// Parses a block sequence, or returns false if there isn't one.
        /// </summary>
        private bool ParseBlockSequence() // [183]
        {
            // Check for the '-' indicator
            if (Token != Token.BlockSeqBegin) return false;
            _load.BeginSeq(null);
            NextToken();
            ParseBlockOrEmptyNode();

            bool recovering = false;

            while (true)
            {
                switch (Token)
                {
                    case Token.EndOfStream:
                    case Token.Outdent:
                        _load.EndSequenceNode();
                        ConsumeOutdent();
                        return true;

                    case Token.BlockSeqNext: // Subsequent '-'s
                        recovering = false;
                        NextToken();
                        ParseBlockOrEmptyNode();
                        continue;

                    case Token.InconsistentIndent:
                        if (!recovering)
                            AddError(Severity.Warning, "YA117", new SourceSpan(_scanner.LineStart, _scanner.TokenEnd));
                        return false;

                    default:
                        if (!recovering)
                            AddError("YA111");
                        recovering = true;
                        NextToken();
                        break;
                }
            }
        }

        /// <summary>
        /// Parses a block-sink node, or generates an empty one
        /// </summary>
        /// <returns>true, just to fit the pattern</returns>
        private bool ParseBlockOrEmptyNode()
        {
            if (!ParseBlockNode())
                ParseEmptyNode();
            return true;

            // Note: this is a lot like BlockIndented [185], without having to 
            // worry about compact notation (which is understood by YamlScanner.
        }


        // The YAML specs compact-sequence and compact-mapping productions
        // aren't needed because the YamlScanner take care of all indentation
        // processing.

        /// <summary>
        /// Parses a YAML block mapping node, or returns false if there isn't one.
        /// </summary>
        private bool ParseBlockMapping() // [187]
        {
            if (Token != Token.BlockMappingBegin) return false;
            _load.BeginMap(null);
            NextToken();
            while (true)
            {
                if (!ParseBlockMapEntry())
                {
                    _scanner.SkipToEOL();
                    return AddError("YA112");
                }
                if (Token == Token.Outdent || Token == Token.EndOfStream)
                    break;
            }
            _load.EndMappingNode();
            if (Token != Token.EndOfStream) ConsumeOutdent();
            return true;
        }

        /// <summary>
        /// Parses the next block mapping entry, or returns false if there isn't one.
        /// </summary>
        private bool ParseBlockMapEntry() // [188]
        {
            switch (Token)
            {
                case Token.BlockKeyIndicator:
                    return ParseBlockMapExplicitEntry();

                case Token.ImplicitKey:
                    return ParseBlockMapImplicitEntry();

                case Token.ValueIndicator:
                    ParseEmptyScalar();
                    return ParseBlockMapImplicitValue();

                default:
                    return false;
            }
        }

        /// <summary>
        /// Parses an explicitly-indicated block mapping entry, or returns false if there isn't one.
        /// </summary>
        /// <returns>true</returns>
        private bool ParseBlockMapExplicitEntry() // [189]
        {
            if (!ParseBlockMapExplicitKey()) return false;

            if (Token == Token.InconsistentIndent)
            {
                AddError("YA117");
                NextToken();
            }
            return ParseBlockMapExplicitValue() || ParseEmptyScalar();
        }

        /// <summary>
        /// Parses an explicitly-indicated block mapping isKey, or returns false if there isn't one.
        /// </summary>
        /// <returns>true</returns>
        private bool ParseBlockMapExplicitKey() // [190]
        {
            if (Token != Token.BlockKeyIndicator) return false;
            NextToken();
            return ParseBlockOrEmptyNode();
        }

        /// <summary>
        /// Parses an explicitly-indicated block mapping value, or returns false if there isn't one.
        /// </summary>
        /// <returns>true</returns>
        private bool ParseBlockMapExplicitValue() // [191]
        {
            if (!MatchToken(Token.ValueIndicator)) return false;

            if (MatchToken(Token.Outdent))
            {
                // Since people perceive the '-' indicator as indentation, block sequences 
                // nested inside a block mapping can skip the indentation.  [200]

                if (Token == Token.BlockSeqBegin)
                    return ParseBlockNode();

                // Otherwise its an empty node, but the outdent has already been consumed,
                // so we call EmptyScalar.

                return ParseEmptyScalar();
            }

            return ParseBlockOrEmptyNode();
        }

        /// <summary>
        /// Parses an implicitly-indicated block mapping entry, or returns false.
        /// </summary>
        private bool ParseBlockMapImplicitEntry() // [192]
        {
            if (!ParseBlockMapImplicitKey())
                ParseEmptyNode();
            return ParseBlockMapImplicitValue();
        }

        /// <summary>
        /// Parses an implicitly-indicated block mapping isKey, or returns false if there isn't one.
        /// </summary>
        private bool ParseBlockMapImplicitKey() // [193]
        {
            // Made so much easier by YamlScanner's doing the lookahead for us...

            if (!MatchToken(Token.ImplicitKey)) return false;
            return ParseFlowJsonNode() || ParseFlowYamlNode();
        }

        /// <summary>
        /// Parses the value of an implicit map entry, or returns false.
        /// </summary>
        /// <returns></returns>
        private bool ParseBlockMapImplicitValue() // [194]
        {
            if (!MatchToken(Token.ValueIndicator))
            {
                // This should really never happen, YamlScanner said a
                // value indicator one was comming, where is it now?
                return InternalError();
            }

            if (ParseBlockNode()) return true; // The normal case

            // If above BlockNode() didn't succeed, there was nothing at the expected
            // indent level, consume the Outdent or we've got a big problem...

            ConsumeOutdent();

            // Since people perceive the '-' indicator as indentation, block sequences 
            // nested inside a block mapping can skip the indentation.  [200]

            if (Token == Token.BlockSeqBegin)
                return ParseBlockNode();

            // Otherwise, nothing followed the ':', so it's an empty node.  
            // The Outdent was already consumed, so just call empty scalar.

            return ParseEmptyScalar();
        }

        private void ConsumeOutdent()
        {
            if (!MatchToken(Token.Outdent))
            {
                AddError("YA101"); // Unexpected content
            }
        }

        /// <summary>
        /// Parses a block-sink node, or returns false. 
        /// </summary>
        /// <remarks>
        /// A flow-sink node is a valid block-sink node.
        /// </remarks>
        private bool ParseBlockNode() // [196]-[199]
        {
            ParseProperties(); // Consume optional properties 

            // Note: properties are consumed above even it isn't a block node,
            // but that's generally OK since we're about to check if it's a flow node,
            // and generally speaking properties are properties, block or flow.

            return ParseLiteral() || ParseFolded() || ParseBlockCollection() || ParseFlowNode();
        }

        // This property is strictly an aide in debugging

        private bool ParseFolded() // [174]
        {
            if (!MatchRawToken(Token.Folded)) return false;
            if (!ParseBreakComment()) return AddError("YA101");

            if (TraceTokens.TraceInfo) Trace.Indent();
            try
            {
                ScalarStart();

                ParseEmptyLines();

                if (_longestEmptyLine >= _scanner.TokenSpan.Start.Column)
                    IssueWarning("YA120"); // Treat it like a warning

                if (Token == Token.TextContent)
                {
                    while (Token != Token.Outdent)
                    {
                        if (!ParseUnindentedFoldedLines()) break;
                        _lineBreaks.Append(NormalizedNewLine);
                        if (!ParseIndentedFoldedLines()) break;
                        ScalarAppend(String.Empty);
                    }
                }
                else if (Token == Token.LeadingWhiteSpace)
                {
                    while (Token != Token.Outdent)
                    {
                        if (!ParseIndentedFoldedLines()) break;
                        ScalarAppend(String.Empty);
                        if (!ParseUnindentedFoldedLines()) break;
                        _lineBreaks.Append(NormalizedNewLine);
                    }
                }
                ScalarChomp();
                return true;
            }
            finally
            {
                if (TraceTokens.TraceInfo) Trace.Unindent();
            }
        }

        private bool ParseIndentedFoldedLines()
        {
            while (Token == Token.LeadingWhiteSpace)
            {
                ScalarAppend(_scanner.TokenBufferSegment);
                if (NextRawToken() == Token.TextContent)
                {
                    ScalarAppend(_scanner.TokenBufferSegment);
                    NextRawToken();
                }
                if (MatchRawToken(Token.Newline))
                {
                    _lineBreaks.Append(NormalizedNewLine);
                }
                ParseEmptyLines();
            }
            return Token != Token.Outdent;
        }

        private bool ParseUnindentedFoldedLines()
        {
            bool addSpace = false;
            while (Token == Token.TextContent)
            {
                if (addSpace)
                {
                    ScalarAppend(' ');
                }
                ScalarAppend(_scanner.TokenBufferSegment);
                NextRawToken();
                if (MatchRawToken(Token.Newline)) addSpace = true;
                if (ParseEmptyLines()) addSpace = false;
            }
            return Token != Token.Outdent;
        }

        private bool ParseEmptyLines()
        {
            if (MatchRawToken(Token.EmptyLine))
                _longestEmptyLine = Math.Max(_longestEmptyLine, TokenSpan.Length);

            int i = 0;
            while (MatchRawToken(Token.Newline))
            {
                _lineBreaks.Append(NormalizedNewLine);
                i++;
                if (MatchRawToken(Token.EmptyLine))
                    _longestEmptyLine = Math.Max(_longestEmptyLine, TokenSpan.Length);
            }
            return i > 0;
        }

        /// <summary>
        /// Parses a block-sink scalar (literal or folded) or returns false.
        /// </summary>
        private bool ParseLiteral() // [170]
        {
            // TODO: strip out the bits for block folded scalars
            if (!MatchRawToken(Token.Literal)) return false;
            if (!ParseBreakComment()) return AddError("YA101");

            if (TraceTokens.TraceInfo) Trace.Indent();
            try
            {
                ScalarStart();
                ParseEmptyLines();
                if (_longestEmptyLine >= _scanner.TokenSpan.Start.Column)
                    IssueWarning("YA120"); // Treat it like a warning

                while (Token != Token.Outdent)
                {
                    switch (Token)
                    {
                        case Token.TextContent:
                            ScalarAppend(_scanner.TokenBufferSegment);
                            break;
                        case Token.Newline:
                            _lineBreaks.Append(NormalizedNewLine);
                            break;
                        case Token.EmptyLine:
                            break;
                        default:
                            return InternalError();
                    }
                    NextRawToken();
                }
                ScalarChomp();
                return true;
            }
            finally
            {
                if (TraceTokens.TraceInfo) Trace.Unindent();
            }
        }

        private void ScalarChomp()
        {
            switch (_scanner.Chomping)
            {
                case '-':
                    break;
                case '+':
                    _scalarBuilder.Append(_lineBreaks.ToString());
                    break;
                case ' ':
                    if (_scalarBuilder.Length != 0)
                        _scalarBuilder.Append(NormalizedNewLine);
                    break;
            }

            _load.AddScalar(_scalarBuilder.ToString());

            if (!MatchToken(Token.Outdent)) InternalError("Missing Outdent");
        }

        private void ScalarStart()
        {
            _scalarBuilder.Length = 0;
            _lineBreaks.Length = 0;
            _longestEmptyLine = 0;
        }

        private void ScalarAppend<T>(T content)
        {
            // Keeping breaks in a seperate place makes
            // chomping easier, but if we're adding more content
            // we have to add them first.
            if (_lineBreaks.Length > 0)
            {
                _scalarBuilder.Append(_lineBreaks.ToString());
                _lineBreaks.Length = 0;
            }
            // Append the content
            _scalarBuilder.Append(content);
        }

        private bool ParseBreakComment()
        {
            MatchRawToken(Token.Comment);
            return MatchRawToken(Token.Newline);
        }


        /// <summary>
        /// Parses a block-sink collection (sequence or mapping)
        /// </summary>
        /// <returns></returns>
        private bool ParseBlockCollection() // [200]
        {
            return ParseBlockSequence() || ParseBlockMapping();
        }

        /// <summary>
        /// Parses a directives-end indicator (---) or returns false.
        /// </summary>
        /// <remarks>
        /// Generates a call to virtual OnDirectivesEnd() if needed.
        /// </remarks>
        private bool ParseDirectivesEnd() // [203]
        {
            if (Token != Token.DirectivesEnd) return false;
            OnDirectivesEnd();
            return MatchToken(Token.DirectivesEnd);
        }

        /// <summary>
        /// Parses a document-end indicator (...), or returns false.
        /// </summary>
        /// <remarks>
        /// Since this indicator is optional, OnDocumentEnd() is called elsewhere.
        /// </remarks>
        private bool ParseDocumentEnd() // [204]
        {
            return MatchToken(Token.DocumentEnd);
        }

        /// <summary>
        /// Parses a document suffix, or returns false.
        /// </summary>
        /// <returns></returns>
        private bool ParseDocumentSuffix() // [205]
        {
            return ParseDocumentEnd(); // Comments handled automatically
        }

        /// <summary>
        /// Parses a YAML document in a stream, or returns false if no content found
        /// </summary>
        /// <remarks>
        /// Calls virtual OnDocumentEnd after content is parsed.
        /// </remarks>
        private bool ParseBareDocument() // [207]
        {
            _load.BeginDocument();
            if (!ParseBlockNode()) return false;
            _load.EndDocument();
            _yamlVersionSetThisDocument = false;
            _shortcutCache.Clear();
            return true;
        }

        /// <summary>
        /// Called to parse an empty document.   Consumes no input.   Always succeeds.
        /// </summary>
        /// <remarks>
        /// Calls virtual OnScalar() with empty content, then OnDocumentEnd().   
        /// </remarks>
        private bool ParseEmptyDocument()
        {
            _load.BeginDocument();
            if (!ParseEmptyScalar()) return false;
            _load.EndDocument();
            _yamlVersionSetThisDocument = false;
            _shortcutCache.Clear();
            return true;
        }

        /// <summary>
        /// Parses an 'explicit' document, e.g. one with a '---', or returns false.
        /// </summary>
        private bool ParseExplicitDocument() // [208]
        {
            return ParseDirectivesEnd() && (ParseBareDocument() || ParseEmptyDocument());
        }

        /// <summary>
        /// Parses a document with leading directives, or returns false if no directives.
        /// </summary>
        private bool ParseDirectivesDocument() // [209]
        {
            if (Token != Token.Directive) return false;

            while (ParseDirective())
            {
            }

            return ParseExplicitDocument();
        }

        /// <summary>
        /// Parses a directive, or returns false.
        /// </summary>
        private bool ParseDirective() // [82]
        {
            if (Token == Token.Directive)
            {
                OnDirective(TokenText);
                NextToken();
                return true;
            }
            return false;
        }

        /// <summary>
        /// Parses any sort of document, with or without prefix, or returns false if none.
        /// </summary>
        /// <returns></returns>
        private bool ParseAnyDocument() // [210]
        {
            return ParseDirectivesDocument() || ParseExplicitDocument() || ParseBareDocument();
        }

        /// <summary>
        /// Parses the next YAML document in the stream.   
        /// </summary>
        /// <returns>true if a document was found, false at end-of-stream</returns>
        public bool ParseDocument() // Effectively [211]
        {
            // YamlScanner handles everything l-document-prefix should, so this
            // becomes a simple 

            if (_firstDocumentInStream)
            {
                _firstDocumentInStream = false;
                return ParseAnyDocument();
            }
            if (ParseDocumentSuffix())
            {
                while (ParseDocumentSuffix())
                {
                } // Multiple end-of-documents in a row ignored
                return ParseAnyDocument();
            }
            return ParseExplicitDocument();
        }

        private bool MatchToken(Token expected)
        {
            if (Token == expected)
            {
                NextToken();
                return true;
            }
            return false;
        }

        private bool MatchRawToken(Token expected)
        {
            if (Token == expected)
            {
                NextRawToken();
                return true;
            }
            return false;
        }


        /// <summary>
        /// Process the contents of a flow scalar, plain, single-quoted, or double-quoted.
        /// </summary>
        /// <param name="end">EndCollection-token expected for this type of scalar</param>
        /// <returns>True if everything went well</returns>
        /// <remarks>
        /// With the assistance of YamlScanner, this function takes care of escapes and 
        /// unfolding of the content per the Yaml spec. at the end, it calls virtual
        /// OnScalar() to deliver the processed value.
        /// </remarks>
        [SuppressMessage("Microsoft.Maintainability", "CA1502:AvoidExcessiveComplexity")]
        private bool ParseFlowScalar(Token end)
        {
            SourceLocation startLocation = _scanner.TokenStart;

            int newlinesPending = 0;
            var sb = new StringBuilder();
            if (TraceTokens.TraceInfo) Trace.Indent();
            try
            {
                while (true)
                {
                    switch (NextRawToken())
                    {
                        case Token.TextContent:
                            if (newlinesPending == 1)
                            {
                                sb.Append(' ');
                            }
                            newlinesPending = 0;
                            sb.Append(_scanner.TokenBufferSegment);
                            continue;

                        case Token.Escape:
                            if (newlinesPending == 1)
                            {
                                sb.Append(' ');
                            }
                            newlinesPending = 0;
                            sb.Append(TokenText);
                            continue;

                        case Token.Newline:
                            if (newlinesPending != 0)
                            {
                                sb.Append(NormalizedNewLine);
                            }
                            newlinesPending++;
                            continue;

                        case Token.EscapedLineBreak:
                        case Token.EscapeWhiteSpace:
                            continue;

                        case Token.PlainEnd:
                        case Token.SingleQuote:
                        case Token.DoubleQuote:
                        case Token.Comment:
                            MatchToken(Token.Comment); // Ignore comment after plain

                            if (!MatchToken(end)) goto case Token.Outdent;

                            if (newlinesPending != 0 && end != Token.PlainEnd) sb.Append(" ");

                            _load.AddScalar(end == Token.PlainEnd
                                                  ? sb.ToString().Trim()
                                                  : sb.ToString());

                            return true;

                        case Token.Outdent:
                            return AddError(end == Token.PlainEnd ? "YA124" : "YA109",
                                            new SourceSpan(startLocation, _scanner.TokenStart));

                        default:
                            return InternalError();
                    }
                }
            }
            finally
            {
                if (TraceTokens.TraceInfo) Trace.Unindent();
            }
        }

        private bool ParseFlowYamlContent() // [156]
        {
            return ParsePlain();
        }

        private bool ParseFlowJsonContent() // [157]
        {
            switch (Token)
            {
                case Token.OpenBracket:
                    return ParseFlowSequence();

                case Token.OpenBrace:
                    return ParseFlowMapping();

                case Token.SingleQuote:
                    return ParseSingleQuoted();

                case Token.DoubleQuote:
                    return ParseDoubleQuoted();

                case Token.ImplicitKey:
                    return ParseFlowPairYamlKeyEntry();

                default:
                    return false;
            }
        }

        private bool ParseFlowContent() // [158]
        {
            return ParseFlowYamlContent() || ParseFlowJsonContent();
        }

        private bool ParseFlowYamlNode() // [159]
        {
            return ParseAliasNode() || ParseFlowYamlContent() ||
                   (ParseProperties() && (ParseFlowYamlContent() || ParseEmptyNode()));
        }

        private bool ParseFlowJsonNode() // [160]
        {
            ParseProperties();
            return ParseFlowJsonContent();
        }

        private bool ParseFlowNode() // [161]
        {
            return ParseAliasNode() || ParseFlowContent()
                   || (ParseProperties() && (ParseFlowContent() || ParseEmptyScalar()));
        }

        private bool InternalError(string message)
        {
            AddError("YA100");
            string formatted = String.Format(CultureInfo.InvariantCulture,
                                             "Internal YamlParser error: {0}", message);
            throw new InvalidOperationException(formatted);
        }

        private bool InternalError()
        {
            return InternalError("Unexpected state");
        }

        #region FlowMappings

        /// <summary>
        /// Parses a flow-sink mapping { ... }, or returns false if none found
        /// </summary>
        private bool ParseFlowMapping() // [140]-[141]
        {
            if (Token != Token.OpenBrace) return false;
            SourceSpan startSpan = _scanner.TokenSpan;
            _load.BeginMap(null);
            NextToken();

            while (ParseFlowMapEntry())
                if (!MatchToken(Token.Comma))
                    break;

            if (!MatchToken(Token.CloseBrace))
            {
                AddError("YA113", new SourceSpan(startSpan.Start, _scanner.TokenEnd));
                AddError("YA114", _scanner.TokenSpan);
            }
            // TODO: OnEndMapping has advanced too far to put out the right tokenText
            _load.EndMappingNode();
            return true;
        }

        /// <summary>
        /// Parses a flow-mapping entry, or returns false is not found.
        /// </summary>
        /// <returns></returns>
        private bool ParseFlowMapEntry() // [142]
        {
            if (Token == Token.CloseBrace)
                return false;

            if (MatchToken(Token.FlowKeyIndicator))
                return ParseFlowMapExplicitEntry();

            MatchToken(Token.ImplicitKey); // Normally present but forge ahead anyway...

            return ParseFlowMapEntryGuts();
        }

        /// <summary>
        /// Parses an explicit flow-mapping entry (one that was preceded with a '?'), even if empty.
        /// </summary>
        /// <returns>true, just to fit the pattern.</returns>
        private bool ParseFlowMapExplicitEntry() // [143] 
        {
            if (!ParseFlowMapEntryGuts())
            {
                // Since there was an explicit '?', but no nodes following it, 
                // synthesize an empty entry..
                ParseEmptyScalar();
                ParseEmptyScalar();
            }
            return true;
        }

        /// <summary>
        /// Parses the inside of a flow-mapping entry, or returns false if not found. 
        /// </summary>
        private bool ParseFlowMapEntryGuts() // [144]
        {
            // Needed rework to avoid lookahead requirement for example 
            // 7.2's funky property-only isKey example:
            //
            // {
            // foo : !!str,
            // !!str : bar,
            // }
            //

            ParseProperties(); // Process optional properties of isKey

            if (ParseFlowMapEmptyKeyEntry()) return true;

            //  Parse isKey portion [145] & [148]
            if (!(ParseFlowJsonContent() || ParseFlowYamlNode()))
                return false;

            // Parse value portion
            ParseProperties(); // Process optional properties of value


            return ParseFlowMapValue() || ParseEmptyScalar();
        }

        /// <summary>
        /// Parses an empty flow-map isKey, or returns false if not found.
        /// </summary>
        /// <returns></returns>
        private bool ParseFlowMapEmptyKeyEntry() // [146]
        {
            // TODO: If this isn't a ':', it's not empty...   or is it????

            if (Token != Token.ValueIndicator) return false;
            ParseEmptyScalar(); // Empty isKey
            ParseFlowMapValue(); // Consume the value indicator and value
            return true;
        }

        /// <summary>
        /// Parses a flow-map value (including the ':') or returns false.
        /// </summary>
        private bool ParseFlowMapValue() // [147] & [149]
        {
            if (!MatchToken(Token.ValueIndicator)) return false;
            if (!ParseFlowNode()) ParseEmptyScalar();
            return true;
        }


        // FlowPair[150] - See FlowSeqEntry() // [139] 

        private void ParseFlowPairEntry() // [151]
        {
            bool dummy =
                ParseFlowPairYamlKeyEntry()
                || ParseFlowMapEmptyKeyEntry()
                || ParseFlowPairJsonKeyEntry();
        }

        private bool ParseFlowPairYamlKeyEntry() // [152] & [154]
        {
            if (ParseImplicitYamlKey())
                return ParseFlowMapValue();
            return false;
        }

        private bool ParseFlowPairJsonKeyEntry() // [153] 
        {
            if (ParseImplicitJsonKey())
                return ParseFlowMapValue();
            return false;
        }

        private bool ParseImplicitYamlKey() // [154] 
        {
            if (!MatchToken(Token.ImplicitKey)) return false;
            return ParseFlowJsonNode() || ParseFlowYamlNode();
        }

        private bool ParseImplicitJsonKey() // [155]
        {
            if (!MatchToken(Token.ImplicitKey)) return false;
            return ParseFlowJsonNode() || ParseFlowYamlNode();
        }

        #endregion

        protected virtual void Dispose(bool disposing)
        {
            if (disposing)
            {
                if (_scanner != null) _scanner.Dispose();
            }
        }

        public void Dispose()
        {
            Dispose(true);
        }
    }
}